//website
Scraper = function(msg){
	this.loadSitemap(msg.sitemap);
	this.meta = msg.meta;
}

Scraper.prototype.loadSitemap = function(sm){
	db2sitemap(sm);
}

Scraper.prototype.startScrape = function(){
	debugger;
	this.log("start ...");
	
	
	var tableData = [];//用来存储爬取得数据
	var links = [];//用来记录所有的链接信息，包括linkedPage和nextPage；
	
	//papaElements, sourcePages, sitemap;
	var startPage = sourcePages[this.meta.sourceIndex];
	//查找主页中每一行记录的字段值
	if(startPage.rowElement && startPage.rowElement.selector){
		var $rows = startPage.rowElement.select();
		var startRow = this.meta.rowIndex;
		for (var r = 0; r < $rows.size(); r++) {
			startRow ++;
			var rowData = {
				rowIndex: this.meta.rowIndex
			};
			var $row = $($rows[r]);
			var fields = startPage.fields;
			for (var i = 0; i < fields.length; i++) {
				var f = fields[i];
				var fvalue = f.getValue($row);
				var fkey = f.id;
				rowData[fkey] = fvalue;
			}
			tableData.push(rowData);
			
			//查找链接页
			for (var i = 0; i < startPage.linkedPages.length; i++) {
				var p = startPage.linkedPages[i];
				var url = p.paging.nextPage($row);
				var obj = {
					sourceIndex: sourcePages.indexOf(p.sourcePage),
					rowIndex: startRow,
					url: url
				};
				links.push(obj);
			}
		}
	}else{
		var rowData = {
			rowIndex: this.meta.rowIndex
		};
		var fields = startPage.fields;
		for (var i = 0; i < fields.length; i++) {
			var f = fields[i];
			var fvalue = f.getValue($row);
			var fkey = f.id;
			rowData[fkey] = fvalue;
		}
		tableData.push(rowData);
	}
	//next page
	if(startPage.paging 
			&& startPage.paging.nextPageElement 
			&& startPage.paging.nextPageElement.selector){
		var url = startPage.paging.nextPage();
		if(url && url.indexOf('http') == 0){
			links.push({
				sourceIndex: this.meta.sourceIndex,
				rowIndex: startRow,
				url: url
			})
		}
	}
	this.sendData(tableData, links, this.meta);
	
	
	this.log("finish ...");
}


Scraper.prototype.sendData = function(data, links, meta){
	chrome.runtime.sendMessage({
		cmd: 'sendData',
		params: {
			data: data,
			links: links,
			meta: meta
		}
	});
}

///*
// * sourceIndex： SourcePage在sitemap.pages的位置
// * pageIndex： 页数，从0开始
// */
//Scraper.prototype.openPage = function(url, sourceIndex, pageIndex){
//	chrome.runtime.sendMessage({
//		cmd: 'openPage',
//		params: {
//			url: url,
//			sourceIndex: sourceIndex,
//			pageIndex: pageIndex
//		}
//	});
//}
//
//Scraper.prototype.closeTab = function(sourceIndex){
//	chrome.runtime.sendMessage({
//		cmd: 'closeTab',
//		params: sourceIndex
//	});
//}


Scraper.prototype.log = function(content){
	chrome.runtime.sendMessage({
		cmd: 'log',
		params: content
	});
}



chrome.runtime.onMessage.addListener(function(msg, _, callback){
	new Scraper(msg).startScrape();
});